{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://github.com/aisingapore/seacorenlp-data/raw/main/id/constituency/train.txt\n",
    "# !wget https://github.com/aisingapore/seacorenlp-data/raw/main/id/constituency/test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-21 10:11:01.212774: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-09-21 10:11:01.291210: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-09-21 10:11:01.733338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-09-21 10:11:01.733370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-09-21 10:11:01.733373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "/home/husein/dev/malaya/malaya/tokenizer.py:214: FutureWarning: Possible nested set at position 3397\n",
      "  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))\n",
      "/home/husein/dev/malaya/malaya/tokenizer.py:214: FutureWarning: Possible nested set at position 3927\n",
      "  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from malaya.function.constituency import evaluate, trees_newline as trees\n",
    "from transformers import AutoTokenizer, T5Config\n",
    "from malaya.torch_model.t5 import T5Constituency\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_TOKEN_MAPPING = {\n",
    "    \"-LRB-\": \"(\",\n",
    "    \"-RRB-\": \")\",\n",
    "    \"-LCB-\": \"{\",\n",
    "    \"-RCB-\": \"}\",\n",
    "    \"-LSB-\": \"[\",\n",
    "    \"-RSB-\": \"]\",\n",
    "    \"``\": '\"',\n",
    "    \"''\": '\"',\n",
    "    \"`\": \"'\",\n",
    "    '«': '\"',\n",
    "    '»': '\"',\n",
    "    '‘': \"'\",\n",
    "    '’': \"'\",\n",
    "    '“': '\"',\n",
    "    '”': '\"',\n",
    "    '„': '\"',\n",
    "    '‹': \"'\",\n",
    "    '›': \"'\",\n",
    "    \"\\u2013\": \"--\", # en dash\n",
    "    \"\\u2014\": \"--\", # em dash\n",
    "    }\n",
    "\n",
    "def process_word(word):\n",
    "    word = word.replace('\\\\/', '/').replace('\\\\*', '*')\n",
    "    # Mid-token punctuation occurs in biomedical text\n",
    "    word = word.replace('-LSB-', '[').replace('-RSB-', ']')\n",
    "    word = word.replace('-LRB-', '(').replace('-RRB-', ')')\n",
    "    if word == \"n't\" and cleaned_words:\n",
    "        cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "        word = \"'t\"\n",
    "    return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "class Vocabulary(object):\n",
    "    def __init__(self):\n",
    "        self.frozen = False\n",
    "        self.values = []\n",
    "        self.indices = {}\n",
    "        self.counts = collections.defaultdict(int)\n",
    "\n",
    "    @property\n",
    "    def size(self):\n",
    "        return len(self.values)\n",
    "\n",
    "    def value(self, index):\n",
    "        assert 0 <= index < len(self.values)\n",
    "        return self.values[index]\n",
    "\n",
    "    def index(self, value):\n",
    "        if not self.frozen:\n",
    "            self.counts[value] += 1\n",
    "\n",
    "        if value in self.indices:\n",
    "            return self.indices[value]\n",
    "\n",
    "        elif not self.frozen:\n",
    "            self.values.append(value)\n",
    "            self.indices[value] = len(self.values) - 1\n",
    "            return self.indices[value]\n",
    "\n",
    "        else:\n",
    "            raise ValueError(\"Unknown value: {}\".format(value))\n",
    "\n",
    "    def index_or_unk(self, value, unk_value):\n",
    "        assert self.frozen\n",
    "        if value in self.indices:\n",
    "            return self.indices[value]\n",
    "        else:\n",
    "            return self.indices[unk_value]\n",
    "\n",
    "    def count(self, value):\n",
    "        return self.counts[value]\n",
    "\n",
    "    def freeze(self):\n",
    "        self.frozen = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_treebank = trees.load_trees('train.txt')\n",
    "train_parse = [tree.convert() for tree in train_treebank]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/nanot5-base-malaysian-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_vocab = Vocabulary()\n",
    "label_vocab.index(())\n",
    "\n",
    "tag_vocab = Vocabulary()\n",
    "START = '<s>'\n",
    "STOP = '</s>'\n",
    "UNK = tokenizer.unk_token\n",
    "TAG_UNK = \"UNK\"\n",
    "tag_vocab.index(START)\n",
    "tag_vocab.index(STOP)\n",
    "tag_vocab.index(TAG_UNK)\n",
    "\n",
    "for tree in train_parse:\n",
    "    nodes = [tree]\n",
    "    while nodes:\n",
    "        node = nodes.pop()\n",
    "        if isinstance(node, trees.InternalParseNode):\n",
    "            label_vocab.index(node.label)\n",
    "            nodes.extend(reversed(node.children))\n",
    "        else:\n",
    "            tag_vocab.index(node.tag)\n",
    "            \n",
    "tag_vocab.freeze()\n",
    "label_vocab.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = T5Config.from_pretrained('mesolitica/nanot5-small-malaysian-cased')\n",
    "config.num_labels = label_vocab.size\n",
    "config.num_tags = tag_vocab.size\n",
    "config.tag_loss_scale = 1.0\n",
    "config.label_vocab = {str(k): v for k, v in label_vocab.indices.items()}\n",
    "config.tag_vocab = tag_vocab.indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of T5Constituency were not initialized from the model checkpoint at mesolitica/nanot5-small-malaysian-cased and are newly initialized: ['encoder_encoder.attn_6.w_vs2', 'encoder_encoder.ff_7.w_2c.bias', 'encoder_encoder.attn_4.w_ks1', 'f_tag.1.a_2', 'encoder_encoder.ff_6.layer_norm.a_2', 'encoder_encoder.attn_0.layer_norm.a_2', 'encoder_encoder.attn_6.layer_norm.b_2', 'f_tag.3.weight', 'encoder_encoder.ff_0.w_2p.bias', 'encoder_encoder.ff_5.w_1p.weight', 'encoder_encoder.attn_2.w_vs1', 'encoder_encoder.attn_3.w_ks2', 'encoder_encoder.ff_1.w_2c.weight', 'encoder_encoder.ff_4.layer_norm.b_2', 'encoder_encoder.attn_3.proj1.weight', 'encoder_encoder.ff_6.w_1c.weight', 'f_tag.0.bias', 'f_label.1.a_2', 'f_tag.1.b_2', 'encoder_encoder.ff_2.layer_norm.b_2', 'encoder_encoder.attn_4.layer_norm.a_2', 'encoder_encoder.ff_4.w_1c.bias', 'encoder_encoder.attn_5.w_ks2', 'f_tag.3.bias', 'encoder_encoder.attn_1.proj1.weight', 'encoder_encoder.attn_4.proj1.weight', 'encoder_encoder.attn_7.layer_norm.a_2', 'encoder_encoder.ff_2.w_2c.bias', 'encoder_encoder.attn_2.w_ks2', 'encoder_encoder.attn_1.w_vs2', 'encoder_encoder.ff_6.w_2c.bias', 'encoder_encoder.attn_7.w_qs2', 'embedding.position_table', 'embedding.layer_norm.a_2', 'encoder_encoder.ff_4.w_1p.bias', 'encoder_encoder.ff_0.w_1p.bias', 'encoder_encoder.attn_3.w_ks1', 'encoder_encoder.attn_0.layer_norm.b_2', 'encoder_encoder.attn_4.w_ks2', 'encoder_encoder.attn_1.layer_norm.b_2', 'encoder_encoder.ff_3.w_1c.bias', 'encoder_encoder.attn_6.w_ks2', 'encoder_encoder.attn_3.proj2.weight', 'encoder_encoder.ff_4.w_2p.bias', 'encoder_encoder.attn_2.proj2.weight', 'encoder_encoder.attn_5.proj1.weight', 'encoder_encoder.attn_0.w_qs2', 'encoder_encoder.attn_1.proj2.weight', 'encoder_encoder.attn_6.w_qs2', 'encoder_encoder.ff_6.w_2p.weight', 'encoder_encoder.attn_0.w_vs2', 'encoder_encoder.ff_6.layer_norm.b_2', 'encoder_encoder.ff_7.layer_norm.b_2', 'encoder_encoder.ff_3.w_1c.weight', 'encoder_encoder.attn_5.w_ks1', 'encoder_encoder.ff_7.w_1p.weight', 'f_label.3.weight', 'encoder_encoder.ff_5.layer_norm.a_2', 'encoder_encoder.ff_1.layer_norm.b_2', 'encoder_encoder.ff_5.w_2c.weight', 'encoder_encoder.attn_6.w_qs1', 'encoder_encoder.attn_1.w_qs2', 'encoder_encoder.ff_3.layer_norm.a_2', 'encoder_encoder.attn_5.w_qs2', 'encoder_encoder.ff_1.w_1c.weight', 'encoder_encoder.ff_3.w_1p.bias', 'encoder_encoder.attn_2.w_qs1', 'encoder_encoder.ff_0.layer_norm.b_2', 'encoder_encoder.attn_3.w_qs2', 'encoder_encoder.ff_0.w_1p.weight', 'encoder_encoder.attn_1.w_ks1', 'encoder_encoder.ff_7.layer_norm.a_2', 'encoder_encoder.ff_7.w_1c.bias', 'encoder_encoder.ff_1.w_2p.bias', 'f_label.3.bias', 'encoder_encoder.ff_4.w_2c.bias', 'encoder_encoder.ff_7.w_1p.bias', 'encoder_encoder.attn_7.w_ks2', 'f_label.0.bias', 'encoder_encoder.ff_5.w_2p.weight', 'encoder_encoder.ff_0.w_2c.weight', 'encoder_encoder.ff_3.w_2p.weight', 'encoder_encoder.ff_7.w_1c.weight', 'encoder_encoder.ff_1.layer_norm.a_2', 'encoder_encoder.attn_1.layer_norm.a_2', 'encoder_encoder.ff_3.w_2c.weight', 'encoder_encoder.attn_5.w_vs1', 'encoder_encoder.attn_0.w_ks1', 'encoder_encoder.ff_2.w_2p.bias', 'encoder_encoder.attn_4.layer_norm.b_2', 'encoder_encoder.ff_7.w_2p.weight', 'encoder_encoder.ff_2.w_1c.weight', 'encoder_encoder.attn_4.proj2.weight', 'encoder_encoder.ff_0.layer_norm.a_2', 'encoder_encoder.attn_7.proj1.weight', 'encoder_encoder.attn_3.w_vs1', 'encoder_encoder.ff_2.layer_norm.a_2', 'encoder_encoder.attn_3.w_vs2', 'encoder_encoder.ff_1.w_2p.weight', 'encoder_encoder.ff_7.w_2c.weight', 'encoder_encoder.attn_4.w_qs1', 'encoder_encoder.attn_1.w_ks2', 'encoder_encoder.ff_6.w_1p.weight', 'embedding.layer_norm.b_2', 'encoder_encoder.ff_1.w_1p.weight', 'encoder_encoder.attn_2.w_ks1', 'encoder_encoder.ff_5.w_2c.bias', 'encoder_encoder.attn_2.w_qs2', 'encoder_encoder.attn_7.w_vs2', 'encoder_encoder.attn_7.w_qs1', 'f_tag.0.weight', 'encoder_encoder.attn_2.layer_norm.b_2', 'encoder_encoder.ff_2.w_1p.weight', 'encoder_encoder.ff_5.layer_norm.b_2', 'encoder_encoder.attn_0.proj2.weight', 'encoder_encoder.ff_6.w_2p.bias', 'encoder_encoder.ff_2.w_1c.bias', 'encoder_encoder.ff_2.w_2c.weight', 'encoder_encoder.attn_6.proj1.weight', 'encoder_encoder.attn_0.w_vs1', 'encoder_encoder.attn_1.w_qs1', 'encoder_encoder.attn_6.w_vs1', 'encoder_encoder.ff_6.w_1p.bias', 'encoder_encoder.ff_6.w_2c.weight', 'encoder_encoder.ff_7.w_2p.bias', 'encoder_encoder.attn_4.w_qs2', 'encoder_encoder.ff_4.w_1p.weight', 'encoder_encoder.ff_0.w_2c.bias', 'encoder_encoder.ff_4.w_2p.weight', 'encoder_encoder.attn_1.w_vs1', 'encoder_encoder.attn_2.w_vs2', 'encoder_encoder.ff_1.w_1c.bias', 'encoder_encoder.ff_5.w_2p.bias', 'encoder_encoder.attn_5.layer_norm.a_2', 'encoder_encoder.ff_3.w_2c.bias', 'encoder_encoder.attn_3.layer_norm.b_2', 'encoder_encoder.attn_4.w_vs1', 'encoder_encoder.attn_7.w_ks1', 'f_label.0.weight', 'encoder_encoder.attn_7.layer_norm.b_2', 'encoder_encoder.ff_3.w_2p.bias', 'encoder_encoder.ff_5.w_1c.bias', 'encoder_encoder.ff_5.w_1p.bias', 'encoder_encoder.attn_6.proj2.weight', 'encoder_encoder.attn_3.w_qs1', 'encoder_encoder.attn_5.w_vs2', 'encoder_encoder.attn_0.w_qs1', 'encoder_encoder.attn_2.proj1.weight', 'encoder_encoder.attn_3.layer_norm.a_2', 'f_label.1.b_2', 'encoder_encoder.ff_4.w_1c.weight', 'encoder_encoder.attn_6.w_ks1', 'encoder_encoder.attn_5.proj2.weight', 'encoder_encoder.attn_4.w_vs2', 'encoder_encoder.attn_7.w_vs1', 'encoder_encoder.ff_6.w_1c.bias', 'encoder_encoder.ff_3.w_1p.weight', 'encoder_encoder.ff_0.w_1c.bias', 'encoder_encoder.attn_6.layer_norm.a_2', 'encoder_encoder.ff_4.layer_norm.a_2', 'encoder_encoder.ff_2.w_1p.bias', 'encoder_encoder.ff_1.w_1p.bias', 'encoder_encoder.attn_0.proj1.weight', 'project_bert.weight', 'encoder_encoder.ff_3.layer_norm.b_2', 'encoder_encoder.attn_0.w_ks2', 'encoder_encoder.attn_5.w_qs1', 'encoder_encoder.attn_7.proj2.weight', 'encoder_encoder.attn_5.layer_norm.b_2', 'encoder_encoder.ff_0.w_1c.weight', 'encoder_encoder.ff_4.w_2c.weight', 'encoder_encoder.ff_0.w_2p.weight', 'encoder_encoder.attn_2.layer_norm.a_2', 'encoder_encoder.ff_2.w_2p.weight', 'encoder_encoder.ff_1.w_2c.bias', 'encoder_encoder.ff_5.w_1c.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = T5Constituency.from_pretrained('mesolitica/nanot5-small-malaysian-cased', config = config)\n",
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainable_parameters = [param for param in model.parameters() if param.requires_grad]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = torch.optim.AdamW(trainable_parameters, lr = 2e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchIndices:\n",
    "    def __init__(self, batch_idxs_np):\n",
    "        self.batch_idxs_np = batch_idxs_np\n",
    "        self.batch_idxs_torch = torch.from_numpy(batch_idxs_np)\n",
    "        self.batch_size = int(1 + np.max(batch_idxs_np))\n",
    "\n",
    "        batch_idxs_np_extra = np.concatenate([[-1], batch_idxs_np, [-1]])\n",
    "        self.boundaries_np = np.nonzero(batch_idxs_np_extra[1:] != batch_idxs_np_extra[:-1])[0]\n",
    "        self.seq_lens_np = self.boundaries_np[1:] - self.boundaries_np[:-1]\n",
    "        assert len(self.seq_lens_np) == self.batch_size\n",
    "        self.max_len = int(np.max(self.boundaries_np[1:] - self.boundaries_np[:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_batch(sentences, golds, subbatch_max_tokens=3000):\n",
    "    lens = [\n",
    "        len(tokenizer.tokenize(' '.join([word for (_, word) in sentence]))) + 2\n",
    "        for sentence in sentences\n",
    "    ]\n",
    "\n",
    "    lens = np.asarray(lens, dtype=int)\n",
    "    lens_argsort = np.argsort(lens).tolist()\n",
    "\n",
    "    num_subbatches = 0\n",
    "    subbatch_size = 1\n",
    "    while lens_argsort:\n",
    "        if (subbatch_size == len(lens_argsort)) or (subbatch_size * lens[lens_argsort[subbatch_size]] > subbatch_max_tokens):\n",
    "            yield [sentences[i] for i in lens_argsort[:subbatch_size]], [golds[i] for i in lens_argsort[:subbatch_size]]\n",
    "            lens_argsort = lens_argsort[subbatch_size:]\n",
    "            num_subbatches += 1\n",
    "            subbatch_size = 1\n",
    "        else:\n",
    "            subbatch_size += 1\n",
    "            \n",
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(\n",
    "            sentence + [pad_int] * (max_sentence_len - len(sentence))\n",
    "        )\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process(sentences, golds = None):\n",
    "    \n",
    "    all_input_ids = []\n",
    "    all_word_start_mask = []\n",
    "    all_word_end_mask = []\n",
    "\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "\n",
    "        tokens = []\n",
    "        word_start_mask = []\n",
    "        word_end_mask = []\n",
    "        tokens.append(START)\n",
    "        word_start_mask.append(1)\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        cleaned_words = []\n",
    "        for _, word in sentence:\n",
    "            cleaned_words.append(process_word(word))\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_start_mask.append(0)\n",
    "                word_end_mask.append(0)\n",
    "            word_start_mask[len(tokens)] = 1\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(STOP)\n",
    "        word_start_mask.append(1)\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        all_input_ids.append(input_ids)\n",
    "        all_word_start_mask.append(word_start_mask)\n",
    "        all_word_end_mask.append(word_end_mask)\n",
    "    \n",
    "    padded = tokenizer.pad({\n",
    "        'input_ids': all_input_ids,\n",
    "    }, return_tensors = 'pt')\n",
    "\n",
    "    all_word_start_mask = torch.from_numpy(np.array(pad_sentence_batch(all_word_start_mask, 0)[0]))\n",
    "    all_word_end_mask = torch.from_numpy(np.array(pad_sentence_batch(all_word_end_mask, 0)[0]))\n",
    "    \n",
    "    padded['sentences'] = sentences\n",
    "    padded['all_word_start_mask'] = all_word_start_mask\n",
    "    padded['all_word_end_mask'] = all_word_end_mask\n",
    "    \n",
    "    packed_len = sum([(len(sentence) + 2) for sentence in sentences])\n",
    "    i = 0\n",
    "    tag_idxs = np.zeros(packed_len, dtype=int)\n",
    "    batch_idxs = np.zeros(packed_len, dtype=int)\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        for (tag, word) in [(START, START)] + sentence + [(STOP, STOP)]:\n",
    "            if golds is not None:\n",
    "                tag_idxs[i] = tag_vocab.index_or_unk(tag, TAG_UNK)\n",
    "            else:\n",
    "                tag_idxs[i] = 0\n",
    "            batch_idxs[i] = snum\n",
    "            i += 1\n",
    "    \n",
    "    batch_idxs = BatchIndices(batch_idxs)\n",
    "    padded['batch_idxs'] = batch_idxs\n",
    "    tag_idxs = torch.from_numpy(tag_idxs)\n",
    "    padded['tag_idxs'] = tag_idxs\n",
    "    \n",
    "    if golds is not None:\n",
    "        gold_tag_idxs = tag_idxs\n",
    "        padded['gold_tag_idxs'] = gold_tag_idxs\n",
    "        padded['golds'] = golds\n",
    "        \n",
    "        \n",
    "    if torch.cuda.is_available():\n",
    "        for k in padded.keys():\n",
    "            if isinstance(padded[k], torch.Tensor):\n",
    "                padded[k] = padded[k].cuda()\n",
    "    \n",
    "        padded['batch_idxs'].batch_idxs_torch = padded['batch_idxs'].batch_idxs_torch.cuda()\n",
    "    \n",
    "    return padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(tensor(256.2531, device='cuda:0', grad_fn=<AddBackward0>), tensor(288.7574, device='cuda:0', grad_fn=<MulBackward0>))\n"
     ]
    }
   ],
   "source": [
    "start_index = 0\n",
    "batch_size = 4\n",
    "batch_trees = train_parse[start_index:start_index + batch_size]\n",
    "batch_sentences = [[(leaf.tag, leaf.word) for leaf in tree.leaves()] for tree in batch_trees]\n",
    "for subbatch_sentences, subbatch_trees in split_batch(batch_sentences, batch_trees):\n",
    "    print(model(**process(subbatch_sentences, subbatch_trees)))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# model(**process(subbatch_sentences)), model(**process(subbatch_sentences, subbatch_trees))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:52<00:00,  9.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 0, loss: 36.82793775939941, dev_fscore: (Recall=17.01, Precision=46.06, FScore=24.84, CompleteMatch=0.00, TaggingAccuracy=61.28)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 1, loss: 20.886111581802368, dev_fscore: (Recall=37.44, Precision=54.97, FScore=44.54, CompleteMatch=1.30, TaggingAccuracy=71.97)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 2, loss: 13.812934432029724, dev_fscore: (Recall=55.19, Precision=58.16, FScore=56.64, CompleteMatch=3.60, TaggingAccuracy=83.97)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:51<00:00,  9.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 3, loss: 9.23275183582306, dev_fscore: (Recall=56.12, Precision=62.07, FScore=58.95, CompleteMatch=4.10, TaggingAccuracy=87.85)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 4, loss: 7.489480774402619, dev_fscore: (Recall=59.44, Precision=64.44, FScore=61.84, CompleteMatch=5.00, TaggingAccuracy=89.75)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 5, loss: 6.364905769348145, dev_fscore: (Recall=61.94, Precision=67.68, FScore=64.68, CompleteMatch=6.10, TaggingAccuracy=90.94)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 6, loss: 5.747362490177155, dev_fscore: (Recall=62.91, Precision=67.99, FScore=65.35, CompleteMatch=8.20, TaggingAccuracy=91.45)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:51<00:00,  9.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 7, loss: 5.227579370737076, dev_fscore: (Recall=64.67, Precision=72.25, FScore=68.25, CompleteMatch=8.90, TaggingAccuracy=91.44)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 8, loss: 4.744117466688156, dev_fscore: (Recall=67.02, Precision=72.43, FScore=69.62, CompleteMatch=10.10, TaggingAccuracy=91.51)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 9, loss: 4.442578368902207, dev_fscore: (Recall=69.26, Precision=71.80, FScore=70.51, CompleteMatch=11.50, TaggingAccuracy=92.05)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 10, loss: 4.069772469758988, dev_fscore: (Recall=68.87, Precision=73.50, FScore=71.11, CompleteMatch=10.90, TaggingAccuracy=92.24)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 11, loss: 3.766999579668045, dev_fscore: (Recall=70.51, Precision=72.69, FScore=71.58, CompleteMatch=11.70, TaggingAccuracy=92.40)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 12, loss: 3.4773665713071824, dev_fscore: (Recall=70.20, Precision=72.73, FScore=71.44, CompleteMatch=13.00, TaggingAccuracy=92.45)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 13, loss: 3.2613371192216873, dev_fscore: (Recall=70.16, Precision=72.43, FScore=71.28, CompleteMatch=11.50, TaggingAccuracy=92.36)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 14, loss: 3.106448167920113, dev_fscore: (Recall=71.02, Precision=73.40, FScore=72.19, CompleteMatch=12.10, TaggingAccuracy=92.33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 15, loss: 2.8761437298059462, dev_fscore: (Recall=70.62, Precision=72.97, FScore=71.78, CompleteMatch=12.60, TaggingAccuracy=92.46)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 16, loss: 2.6546426833868026, dev_fscore: (Recall=72.12, Precision=72.36, FScore=72.24, CompleteMatch=12.00, TaggingAccuracy=92.80)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 17, loss: 2.4989955403208732, dev_fscore: (Recall=73.08, Precision=72.30, FScore=72.69, CompleteMatch=13.00, TaggingAccuracy=92.61)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 18, loss: 2.329909897565842, dev_fscore: (Recall=74.13, Precision=73.13, FScore=73.63, CompleteMatch=14.20, TaggingAccuracy=92.88)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 19, loss: 2.261778307080269, dev_fscore: (Recall=73.75, Precision=73.11, FScore=73.43, CompleteMatch=14.10, TaggingAccuracy=92.86)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 20, loss: 2.08204574072361, dev_fscore: (Recall=74.64, Precision=73.25, FScore=73.94, CompleteMatch=14.10, TaggingAccuracy=93.00)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 21, loss: 1.9595084832310676, dev_fscore: (Recall=73.96, Precision=74.75, FScore=74.35, CompleteMatch=14.40, TaggingAccuracy=92.91)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 22, loss: 1.8606698272228241, dev_fscore: (Recall=73.97, Precision=73.90, FScore=73.93, CompleteMatch=14.70, TaggingAccuracy=92.90)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 23, loss: 1.7471688374876977, dev_fscore: (Recall=73.22, Precision=75.66, FScore=74.42, CompleteMatch=13.80, TaggingAccuracy=92.92)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 24, loss: 1.6766351570785045, dev_fscore: (Recall=75.38, Precision=71.95, FScore=73.63, CompleteMatch=13.60, TaggingAccuracy=93.12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:51<00:00,  9.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 25, loss: 1.5632741984128953, dev_fscore: (Recall=74.19, Precision=73.51, FScore=73.85, CompleteMatch=15.10, TaggingAccuracy=92.79)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 26, loss: 1.469447272092104, dev_fscore: (Recall=73.04, Precision=69.55, FScore=71.25, CompleteMatch=11.70, TaggingAccuracy=93.18)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 27, loss: 1.4365589104294776, dev_fscore: (Recall=75.05, Precision=72.94, FScore=73.98, CompleteMatch=14.30, TaggingAccuracy=92.92)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 28, loss: 1.3704663390517235, dev_fscore: (Recall=75.00, Precision=73.00, FScore=73.98, CompleteMatch=13.40, TaggingAccuracy=93.11)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 29, loss: 1.3296874136030674, dev_fscore: (Recall=72.90, Precision=75.47, FScore=74.16, CompleteMatch=14.20, TaggingAccuracy=93.31)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 30, loss: 1.246569149762392, dev_fscore: (Recall=73.68, Precision=72.89, FScore=73.28, CompleteMatch=13.80, TaggingAccuracy=93.38)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 31, loss: 1.2012923715412618, dev_fscore: (Recall=74.87, Precision=76.71, FScore=75.78, CompleteMatch=16.80, TaggingAccuracy=93.17)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 32, loss: 1.108302319675684, dev_fscore: (Recall=74.02, Precision=66.30, FScore=69.95, CompleteMatch=10.40, TaggingAccuracy=93.04)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 33, loss: 1.1320208514630794, dev_fscore: (Recall=75.44, Precision=76.74, FScore=76.08, CompleteMatch=16.10, TaggingAccuracy=93.34)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 34, loss: 1.105076265245676, dev_fscore: (Recall=73.90, Precision=77.03, FScore=75.43, CompleteMatch=15.20, TaggingAccuracy=93.24)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 35, loss: 1.0395332059562206, dev_fscore: (Recall=73.55, Precision=77.24, FScore=75.35, CompleteMatch=15.00, TaggingAccuracy=93.12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 36, loss: 1.0340544445514679, dev_fscore: (Recall=75.50, Precision=77.22, FScore=76.35, CompleteMatch=15.80, TaggingAccuracy=93.38)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 37, loss: 1.0261862611621617, dev_fscore: (Recall=76.06, Precision=75.62, FScore=75.84, CompleteMatch=15.90, TaggingAccuracy=92.91)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:51<00:00,  9.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 38, loss: 0.9706435385942459, dev_fscore: (Recall=75.61, Precision=73.34, FScore=74.46, CompleteMatch=14.90, TaggingAccuracy=93.32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 39, loss: 0.9711148667186499, dev_fscore: (Recall=74.85, Precision=75.74, FScore=75.29, CompleteMatch=15.50, TaggingAccuracy=93.45)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 40, loss: 0.9080727861970663, dev_fscore: (Recall=77.02, Precision=74.46, FScore=75.72, CompleteMatch=15.20, TaggingAccuracy=93.92)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 41, loss: 0.9279245437681675, dev_fscore: (Recall=75.01, Precision=75.97, FScore=75.49, CompleteMatch=14.80, TaggingAccuracy=93.61)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 42, loss: 0.9002708364725113, dev_fscore: (Recall=74.61, Precision=76.21, FScore=75.40, CompleteMatch=15.90, TaggingAccuracy=93.52)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 43, loss: 0.8787384810745716, dev_fscore: (Recall=75.12, Precision=76.79, FScore=75.94, CompleteMatch=15.60, TaggingAccuracy=93.60)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 44, loss: 0.8334691937714815, dev_fscore: (Recall=75.78, Precision=75.81, FScore=75.80, CompleteMatch=15.60, TaggingAccuracy=93.48)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 45, loss: 0.8634952803701162, dev_fscore: (Recall=74.35, Precision=75.68, FScore=75.01, CompleteMatch=15.80, TaggingAccuracy=93.69)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 46, loss: 0.7918994451239705, dev_fscore: (Recall=77.02, Precision=75.96, FScore=76.49, CompleteMatch=17.30, TaggingAccuracy=93.72)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 47, loss: 0.7778680425137281, dev_fscore: (Recall=74.82, Precision=77.14, FScore=75.96, CompleteMatch=15.50, TaggingAccuracy=93.57)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 48, loss: 0.8096335969567299, dev_fscore: (Recall=76.90, Precision=77.06, FScore=76.98, CompleteMatch=17.00, TaggingAccuracy=93.54)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 49, loss: 0.7743520020693541, dev_fscore: (Recall=77.69, Precision=72.61, FScore=75.07, CompleteMatch=14.80, TaggingAccuracy=93.32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 50, loss: 0.7304364721029997, dev_fscore: (Recall=73.94, Precision=77.95, FScore=75.89, CompleteMatch=15.00, TaggingAccuracy=93.58)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 51, loss: 0.7239310853779316, dev_fscore: (Recall=78.74, Precision=74.70, FScore=76.67, CompleteMatch=15.30, TaggingAccuracy=93.47)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 52, loss: 0.7262997675463557, dev_fscore: (Recall=76.69, Precision=78.10, FScore=77.39, CompleteMatch=16.50, TaggingAccuracy=94.03)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 53, loss: 0.6888199557289482, dev_fscore: (Recall=76.49, Precision=76.08, FScore=76.28, CompleteMatch=15.90, TaggingAccuracy=93.49)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 54, loss: 0.6655678783580661, dev_fscore: (Recall=77.58, Precision=76.12, FScore=76.85, CompleteMatch=15.40, TaggingAccuracy=93.47)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 55, loss: 0.6549655245244503, dev_fscore: (Recall=76.06, Precision=76.94, FScore=76.50, CompleteMatch=16.10, TaggingAccuracy=93.63)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 56, loss: 0.6641534368023276, dev_fscore: (Recall=76.81, Precision=76.66, FScore=76.74, CompleteMatch=15.90, TaggingAccuracy=93.56)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 57, loss: 0.6445835774838924, dev_fscore: (Recall=77.56, Precision=77.18, FScore=77.37, CompleteMatch=17.50, TaggingAccuracy=93.81)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 58, loss: 0.6209825110211968, dev_fscore: (Recall=77.54, Precision=77.42, FScore=77.48, CompleteMatch=17.40, TaggingAccuracy=93.68)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 59, loss: 0.6274297954514623, dev_fscore: (Recall=77.61, Precision=77.42, FScore=77.52, CompleteMatch=18.10, TaggingAccuracy=93.68)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 60, loss: 0.5927253080531955, dev_fscore: (Recall=77.86, Precision=77.41, FScore=77.63, CompleteMatch=18.20, TaggingAccuracy=94.02)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 61, loss: 0.6012893799990415, dev_fscore: (Recall=77.69, Precision=77.11, FScore=77.40, CompleteMatch=16.80, TaggingAccuracy=93.74)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 62, loss: 0.6154908737055957, dev_fscore: (Recall=77.53, Precision=76.46, FScore=76.99, CompleteMatch=18.60, TaggingAccuracy=93.62)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 63, loss: 0.5833988428898156, dev_fscore: (Recall=77.84, Precision=77.42, FScore=77.63, CompleteMatch=18.10, TaggingAccuracy=93.68)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 64, loss: 0.5716876466348767, dev_fscore: (Recall=76.87, Precision=77.30, FScore=77.08, CompleteMatch=17.20, TaggingAccuracy=93.82)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 65, loss: 0.5410758772045374, dev_fscore: (Recall=77.79, Precision=77.89, FScore=77.84, CompleteMatch=17.50, TaggingAccuracy=93.83)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 66, loss: 0.5576500458940864, dev_fscore: (Recall=77.44, Precision=79.06, FScore=78.24, CompleteMatch=19.40, TaggingAccuracy=93.40)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 67, loss: 0.5087779211271555, dev_fscore: (Recall=78.36, Precision=76.81, FScore=77.58, CompleteMatch=16.60, TaggingAccuracy=93.87)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 68, loss: 0.5259088312350213, dev_fscore: (Recall=77.94, Precision=78.42, FScore=78.18, CompleteMatch=18.60, TaggingAccuracy=93.63)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 69, loss: 0.4899563212133944, dev_fscore: (Recall=78.51, Precision=77.12, FScore=77.81, CompleteMatch=17.50, TaggingAccuracy=93.55)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 70, loss: 0.4951904037371278, dev_fscore: (Recall=78.62, Precision=79.00, FScore=78.81, CompleteMatch=19.20, TaggingAccuracy=93.87)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 71, loss: 0.5097951237950474, dev_fscore: (Recall=77.33, Precision=79.43, FScore=78.37, CompleteMatch=19.00, TaggingAccuracy=93.73)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 72, loss: 0.49706350382603703, dev_fscore: (Recall=76.47, Precision=79.36, FScore=77.89, CompleteMatch=18.00, TaggingAccuracy=93.68)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 73, loss: 0.4620273124370724, dev_fscore: (Recall=78.93, Precision=77.72, FScore=78.32, CompleteMatch=18.60, TaggingAccuracy=94.10)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 74, loss: 0.421369884878397, dev_fscore: (Recall=78.09, Precision=78.11, FScore=78.10, CompleteMatch=18.50, TaggingAccuracy=93.98)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 75, loss: 0.42392951850406824, dev_fscore: (Recall=79.29, Precision=78.47, FScore=78.88, CompleteMatch=19.50, TaggingAccuracy=93.61)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 76, loss: 0.4463566712774336, dev_fscore: (Recall=78.07, Precision=79.32, FScore=78.69, CompleteMatch=19.20, TaggingAccuracy=93.83)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 77, loss: 0.418001589294523, dev_fscore: (Recall=77.74, Precision=78.84, FScore=78.28, CompleteMatch=18.80, TaggingAccuracy=93.78)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 78, loss: 0.45920864765159786, dev_fscore: (Recall=77.02, Precision=79.45, FScore=78.22, CompleteMatch=18.20, TaggingAccuracy=93.56)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 79, loss: 0.4212995104379952, dev_fscore: (Recall=79.19, Precision=78.39, FScore=78.79, CompleteMatch=18.00, TaggingAccuracy=94.11)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 80, loss: 0.3768032774925232, dev_fscore: (Recall=78.28, Precision=78.17, FScore=78.22, CompleteMatch=18.30, TaggingAccuracy=94.02)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 81, loss: 0.36842332554236057, dev_fscore: (Recall=76.53, Precision=80.39, FScore=78.41, CompleteMatch=18.00, TaggingAccuracy=93.72)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 82, loss: 0.3783338834755123, dev_fscore: (Recall=78.88, Precision=79.21, FScore=79.05, CompleteMatch=18.90, TaggingAccuracy=93.69)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:50<00:00,  9.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 83, loss: 0.36091401127539574, dev_fscore: (Recall=78.42, Precision=80.28, FScore=79.34, CompleteMatch=21.20, TaggingAccuracy=94.12)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 84, loss: 0.3451699822731316, dev_fscore: (Recall=78.70, Precision=79.62, FScore=79.16, CompleteMatch=19.50, TaggingAccuracy=93.79)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 85, loss: 0.35854637472890316, dev_fscore: (Recall=78.56, Precision=79.42, FScore=78.99, CompleteMatch=19.20, TaggingAccuracy=93.90)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 86, loss: 0.3492727878820151, dev_fscore: (Recall=78.77, Precision=79.33, FScore=79.05, CompleteMatch=19.40, TaggingAccuracy=93.93)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 87, loss: 0.33016869584470987, dev_fscore: (Recall=77.57, Precision=80.06, FScore=78.80, CompleteMatch=18.90, TaggingAccuracy=93.87)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 88, loss: 0.32659329972974954, dev_fscore: (Recall=79.06, Precision=78.88, FScore=78.97, CompleteMatch=19.50, TaggingAccuracy=94.27)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 89, loss: 0.3043060428816825, dev_fscore: (Recall=78.84, Precision=79.13, FScore=78.98, CompleteMatch=18.20, TaggingAccuracy=93.88)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 90, loss: 0.2989499959871173, dev_fscore: (Recall=77.88, Precision=80.16, FScore=79.00, CompleteMatch=20.90, TaggingAccuracy=94.15)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 91, loss: 0.3137446914128959, dev_fscore: (Recall=76.53, Precision=81.54, FScore=78.96, CompleteMatch=18.10, TaggingAccuracy=93.88)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 92, loss: 0.2886410137396306, dev_fscore: (Recall=79.02, Precision=79.87, FScore=79.44, CompleteMatch=19.00, TaggingAccuracy=94.18)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 93, loss: 0.29784207583055833, dev_fscore: (Recall=78.28, Precision=79.96, FScore=79.11, CompleteMatch=18.80, TaggingAccuracy=93.56)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 94, loss: 0.29370920681767165, dev_fscore: (Recall=77.41, Precision=80.84, FScore=79.09, CompleteMatch=19.80, TaggingAccuracy=93.95)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 95, loss: 0.30497036553919316, dev_fscore: (Recall=78.11, Precision=80.73, FScore=79.40, CompleteMatch=20.00, TaggingAccuracy=94.15)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 96, loss: 0.2824800045103766, dev_fscore: (Recall=75.49, Precision=80.23, FScore=77.79, CompleteMatch=19.80, TaggingAccuracy=93.89)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 97, loss: 0.29544836299930466, dev_fscore: (Recall=76.96, Precision=80.66, FScore=78.76, CompleteMatch=19.00, TaggingAccuracy=93.99)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 98, loss: 0.2811927164471708, dev_fscore: (Recall=79.10, Precision=80.63, FScore=79.86, CompleteMatch=20.10, TaggingAccuracy=93.96)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:49<00:00, 10.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "epoch: 99, loss: 0.2393738731777994, dev_fscore: (Recall=78.56, Precision=80.29, FScore=79.41, CompleteMatch=20.20, TaggingAccuracy=94.09)\n"
     ]
    }
   ],
   "source": [
    "batch_size = 16\n",
    "epoch = 100\n",
    "\n",
    "best_dev_fscore = -np.inf\n",
    "patient = 10\n",
    "current_patient = 0\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm(range(0, len(train_parse), batch_size))\n",
    "    losses = []\n",
    "    for start_index in pbar:\n",
    "        trainer.zero_grad()\n",
    "        batch_loss_value = 0.0\n",
    "        batch_trees = train_parse[start_index:start_index + batch_size]\n",
    "        batch_sentences = [[(leaf.tag, leaf.word) for leaf in tree.leaves()] for tree in batch_trees]\n",
    "        batch_num_tokens = sum(len(sentence) for sentence in batch_sentences)\n",
    "\n",
    "        for subbatch_sentences, subbatch_trees in split_batch(batch_sentences, batch_trees):\n",
    "            loss, tag_loss =  model(**process(subbatch_sentences, subbatch_trees))\n",
    "            loss = tag_loss / len(subbatch_sentences) + loss / batch_num_tokens\n",
    "            loss_value = float(loss.data.cpu().numpy())\n",
    "            batch_loss_value += loss_value\n",
    "            if loss_value > 0:\n",
    "                loss.backward()\n",
    "        \n",
    "        grad_norm = torch.nn.utils.clip_grad_norm_(trainable_parameters, 5.0)\n",
    "        trainer.step()\n",
    "        losses.append(batch_loss_value)\n",
    "        \n",
    "    dev_predicted = []\n",
    "    for dev_start_index in range(0, len(dev_treebank), batch_size):\n",
    "        subbatch_trees = dev_treebank[dev_start_index:dev_start_index+batch_size]\n",
    "        subbatch_sentences = [[(leaf.tag, leaf.word) for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "        predicted, _ =  model(**process(subbatch_sentences))\n",
    "        dev_predicted.extend([p.convert() for p in predicted])\n",
    "    \n",
    "    dev_fscore = evaluate.evalb('deprecated/EVALB', dev_treebank, dev_predicted)\n",
    "    \n",
    "    print(f'epoch: {e}, loss: {np.mean(losses)}, dev_fscore: {dev_fscore}')\n",
    "    \n",
    "    if dev_fscore.fscore > best_dev_fscore:\n",
    "        best_dev_fscore = dev_fscore.fscore\n",
    "        current_patient = 0\n",
    "        model.save_pretrained('small')\n",
    "    else:\n",
    "        current_patient += 1\n",
    "    \n",
    "    if current_patient >= patient:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ = T5Constituency.from_pretrained('./small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "854d60c567614ce5a9b4b63c1f930820",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/273M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/constituency-parsing-nanot5-small-malaysian-cased/commit/15f85af31523aed2eac7d3fedd5e1dcc981448c3', commit_message='Upload T5Constituency', commit_description='', oid='15f85af31523aed2eac7d3fedd5e1dcc981448c3', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_.push_to_hub('mesolitica/constituency-parsing-nanot5-small-malaysian-cased', safe_serialization = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/constituency-parsing-nanot5-small-malaysian-cased/commit/be911c3f2acfa342a9f95ffed3c3d224435e33a4', commit_message='Upload tokenizer', commit_description='', oid='be911c3f2acfa342a9f95ffed3c3d224435e33a4', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('mesolitica/constituency-parsing-nanot5-small-malaysian-cased', safe_serialization = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
